# -*- coding: utf-8 -*-

import scrapy
#from test1.items import jobinfoitem, cominfoitem
from test1.items import jobinfoitem, cominfoitem,mittcheliang
from scrapy import Request
from scrapy.loader import ItemLoader
from scrapyluke.processors import *
import re
import time
import datetime

class ChinahrSpider(scrapy.Spider):
    name = 'cheliang'
    cominfo = cominfoitem()
    start_urls = ['http://shenpi.miit.gov.cn/commonquery/result?categoryId=327&pagenow=1']

    def parse(self,response):
        if response.xpath('//input[@name="aId"]/@value'):
            #利用xpath获取各公司的aid
            job_url= response.xpath('//input[@name="aId"]/@value').extract()
            for aidnum in job_url:
                aidnum2 = str(aidnum)
                #利用aid发送请求
                yield scrapy.FormRequest("http://shenpi.miit.gov.cn/commonquery/showDetail",
                                   formdata={'aid':aidnum2},
                                   callback=self.parse_info)
            page = re.findall('oryId=327&pagenow=(\d+)', response.url)[0]
            #翻页
            curr_page = int(page) + 1
            yield scrapy.Request('http://shenpi.miit.gov.cn/commonquery/result?categoryId=327&pagenow=%d' % curr_page, callback= self.parse)

    def parse_info(self, response):
         if response.body:
            cheliang_info = mittcheliang()
            getbody = response.body
            body_list = list(eval(getbody))
            for dict in body_list:                     #判断是否存在相关内容,并放入item(这网站所有的字典的key值居然都相同，是正常人写的么？！)
                checkstr = dict['deffieldName']
                databuf = dict['deffieldValue']
                if '企业名称' in checkstr:
                    cheliang_info['the_name_of_the_enterprise'] = databuf
                elif '商标' in checkstr:
                    cheliang_info['trademark'] = databuf
                elif '产品名称' in checkstr:
                    cheliang_info['product_name'] = databuf
                elif '目录' in checkstr:
                    cheliang_info['serial_number'] = databuf
                elif '产品型号' in checkstr:
                    cheliang_info['product_model'] = databuf
                elif '变更内容' in checkstr:
                    cheliang_info['change_content'] = databuf
                elif '扩展内容' in checkstr:
                    cheliang_info['extended_content'] = databuf
                elif '备注' in checkstr:
                    cheliang_info['remarks'] = databuf
            time = datetime.datetime.now()
            curr_time = str(time)
            cheliang_info['insert_time'] = curr_time
            cheliang_info['url'] = response.url
            return cheliang_info
